import tushare as ts
import pandas as pd
import os
import platform
print('python v', platform.python_version())

# 1. 获取所有股票列表
stock_list_df = ts.get_stock_basics()

# 如果集合的最新日期不等于最近的一个交易日，就以最新日期为start，以当前日期为end获取当前股票历史数据，并且更新数据库，插入新记录

# dd/mm/yyyy格式
# print (time.strftime("%Y-%m-%d"))
# datetime.datetime.now()
# count = 0
# D:/Projects/Test/stocks/trading-data@full.20170717/stock_data/
# G:/20170717/stock_data/
# E:/201709081d&1m/1d/
# D:/new_tdx/T0002/export/1d/
# D:/new_tdx/T0002/export/1m/
stock_data_path = input('Please input stock data directory path: ')

total_count = 0
imported = []
no_data = []
h5w = pd.HDFStore('stocks_1m.h5', 'a', complevel=9, complib='zlib')
# dateparse = lambda dates: pd.datetime.strptime(dates, '%Y/%m/%d%H%M%S')


def dateparse (dates, times):
    return pd.datetime.strptime(dates + ' ' + times, '%Y-%m-%d %H%M')
# 遍历股票列表并且存入数据库
for stock_info in stock_list_df.iterrows():   # 获取每行的index、row
    item = stock_info
    code = item[0]
    file_name = stock_data_path + str('SH#' if code[0] == '6' else 'SZ#') + code + '.csv'
    if os.path.exists(file_name):
        days_df = pd.read_csv(
            file_name,
            skiprows=2,
            skipfooter=1,
            parse_dates=[[0, 1]],
            index_col=0,
            date_parser=dateparse,
            encoding='gbk',
            keep_date_col=False,
            engine='python',
            names=['date', 'time', 'open', 'high', 'low', 'close', 'volume', 'turnover']
        )

        # h5w['stock_1d_' + item["code"]] = days_df
        h5w['stock_' + code] = days_df
        imported.append(code)
        total_count += len(days_df)
        # print(code, '导入 ', len(days_df), '条数据')  # days_df[0, len(days_df) - 1], '-', days_df[0, 0], '共',
        print(code, '导入 ', len(days_df), '条数据')
    else:
        no_data.append(code)

h5w.close()
print('共导入历史数据', total_count, '条')
print(imported)
print(len(no_data), '历史数据不存在')
print(no_data)



